from __future__ import print_function
import sys
from operator import add
from pyspark.sql import SparkSession
from pyspark import SparkContext
from pyspark.conf import SparkConf
"""
if __name__ == "__main__":
    if len(sys.argv) != 2:
        print("Usage: wordcount <file>", file=sys.stderr)
        exit(-1)
    print("-------------------------11111111111----------------------------------")
    spark = SparkSession\
        .builder\
        .appName("PythonWordCount")\
        .getOrCreate()
    print("-------------------------22222222222----------------------------------")
    lines = spark.read.text(sys.argv[1]).rdd.map(lambda r: r[0])
    print("-------------------------33333333333----------------------------------")
    counts = lines.flatMap(lambda x: x.split(' ')) \
                  .map(lambda x: (x, 1)) \
                  .reduceByKey(add)
    print("-------------------------44444444444----------------------------------")
    output = counts.collect()
    print("-------------------------55555555555----------------------------------")
    for (word, count) in output:
        print("%s: %i" % (word, count))
    counts.saveAsTextFile("output")
    print("-------------------------66666666666----------------------------------")
    spark.stop()
"""


def show_result(couple):
	print(couple)

if __name__=='__main__':
    conf = SparkConf()
    conf.setMaster("local")
    conf.setAppName("test")
    print("-------------------------11111111111----------------------------------")
    sc = SparkContext(conf=conf)
	#textFile = sc.textFile("./word")
    textFile = sc.textFile('file:///workspace/bdkit-demo/spark-python-demo/src/total_loan.csv')
    word_count = textFile.flatMap(lambda line: line.split(" ")).map(lambda word: ((str(int(int(word)/1000))),1)).reduceByKey(lambda a, b : a + b)
    #word_count.foreach(lambda word: show_result(word))
    print("--------------------------------------------------------")
    print(type(word_count))
    output = word_count.collect()
    print(type(output))
    print("--------------------------------------------------------")
    for (word, count) in output:
        wordstr='('+str(int(word)*1000)+','+str(int(word)*1000+1000)+')'
        print("%s, %i)" % (wordstr, count))
    spark.stop()
